{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.3、5.4  PyTorchでDQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n"
     ]
    }
   ],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import gym\n",
    "\n",
    "print(\"hello\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 動画の描画関数の宣言\n",
    "# 参考URL http://nbviewer.jupyter.org/github/patrickmineault\n",
    "# /xcorr-notebooks/blob/master/Render%20OpenAI%20gym%20as%20GIF.ipynb\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "from matplotlib import animation\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    \"\"\"\n",
    "    Displays a list of frames as a gif, with controls\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(frames[0].shape[1]/72.0, frames[0].shape[0]/72.0),\n",
    "               dpi=72)\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "\n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames=len(frames),\n",
    "                                   interval=50)\n",
    "\n",
    "    anim.save('movie_cartpole_DQN.mp4')  # 動画のファイル名と保存です\n",
    "    display(display_animation(anim, default_mode='loop'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tr(name_a='名前Aです', value_b=100)\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "# 本コードでは、namedtupleを使用します。\n",
    "# namedtupleを使うことで、値をフィールド名とペアで格納できます。\n",
    "# すると値に対して、フィールド名でアクセスできて便利です。\n",
    "# https://docs.python.jp/3/library/collections.html#collections.namedtuple\n",
    "# 以下は使用例です\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "Tr = namedtuple('tr', ('name_a', 'value_b'))\n",
    "Tr_object = Tr('名前Aです', 100)\n",
    "\n",
    "print(Tr_object)  # 出力：tr(name_a='名前Aです', value_b=100)\n",
    "print(Tr_object.value_b)  # 出力：100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# namedtupleを生成\n",
    "from collections import namedtuple\n",
    "\n",
    "Transition = namedtuple(\n",
    "    'Transition', ('state', 'action', 'next_state', 'reward'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定数の設定\n",
    "ENV = 'CartPole-v1'  # 使用する課題名\n",
    "GAMMA = 0.9  # 時間割引率\n",
    "MAX_STEPS = 200  # 1試行のstep数\n",
    "NUM_EPISODES = 1000  # 最大試行回数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 経験を保存するメモリクラスを定義します\n",
    "\n",
    "\n",
    "class ReplayMemory:\n",
    "\n",
    "    def __init__(self, CAPACITY):\n",
    "        self.capacity = CAPACITY  # メモリの最大長さ\n",
    "        self.memory = []  # 経験を保存する変数\n",
    "        self.index = 0  # 保存するindexを示す変数\n",
    "\n",
    "    def push(self, state, action, state_next, reward):\n",
    "        '''transition = (state, action, state_next, reward)をメモリに保存する'''\n",
    "\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)  # メモリが満タンでないときは足す\n",
    "\n",
    "        # namedtupleのTransitionを使用し、値とフィールド名をペアにして保存します\n",
    "        self.memory[self.index] = Transition(state, action, state_next, reward)\n",
    "\n",
    "        self.index = (self.index + 1) % self.capacity  # 保存するindexを1つずらす\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        '''batch_size分だけ、ランダムに保存内容を取り出す'''\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        '''関数lenに対して、現在の変数memoryの長さを返す'''\n",
    "        return len(self.memory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# エージェントが持つ脳となるクラスです、DQNを実行します\n",
    "# Q関数をディープラーニングのネットワークをクラスとして定義\n",
    "\n",
    "import random\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import warnings\n",
    "# 忽略特定的警告信息\n",
    "# warnings.filterwarnings(\"ignore\", category=torch.jit.TracerWarning)\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "BATCH_SIZE = 32\n",
    "CAPACITY = 10000\n",
    "\n",
    "\n",
    "class Brain:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        self.num_actions = num_actions  # CartPoleの行動（右に左に押す）の2を取得\n",
    "\n",
    "        # 経験を記憶するメモリオブジェクトを生成\n",
    "        self.memory = ReplayMemory(CAPACITY)\n",
    "\n",
    "        # ニューラルネットワークを構築\n",
    "        self.model = nn.Sequential()\n",
    "        self.model.add_module('fc1', nn.Linear(num_states, 32))\n",
    "        self.model.add_module('relu1', nn.ReLU())\n",
    "        self.model.add_module('fc2', nn.Linear(32, 32))\n",
    "        self.model.add_module('relu2', nn.ReLU())\n",
    "        self.model.add_module('fc3', nn.Linear(32, num_actions))\n",
    "\n",
    "        print(self.model)  # ネットワークの形を出力\n",
    "\n",
    "        # 最適化手法の設定\n",
    "        self.optimizer = optim.Adam(self.model.parameters(), lr=0.0001)\n",
    "\n",
    "    def replay(self):\n",
    "        '''Experience Replayでネットワークの結合パラメータを学習'''\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 1. メモリサイズの確認\n",
    "        # -----------------------------------------\n",
    "        # 1.1 メモリサイズがミニバッチより小さい間は何もしない\n",
    "        if len(self.memory) < BATCH_SIZE:\n",
    "            return\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 2. ミニバッチの作成\n",
    "        # -----------------------------------------\n",
    "        # 2.1 メモリからミニバッチ分のデータを取り出す\n",
    "        transitions = self.memory.sample(BATCH_SIZE)\n",
    "\n",
    "        # 2.2 各変数をミニバッチに対応する形に変形\n",
    "        # transitionsは1stepごとの(state, action, state_next, reward)が、BATCH_SIZE分格納されている\n",
    "        # つまり、(state, action, state_next, reward)×BATCH_SIZE\n",
    "        # これをミニバッチにしたい。つまり\n",
    "        # (state×BATCH_SIZE, action×BATCH_SIZE, state_next×BATCH_SIZE, reward×BATCH_SIZE)にする\n",
    "        batch = Transition(*zip(*transitions))\n",
    "\n",
    "        # 2.3 各変数の要素をミニバッチに対応する形に変形し、ネットワークで扱えるようVariableにする\n",
    "        # 例えばstateの場合、[torch.FloatTensor of size 1x4]がBATCH_SIZE分並んでいるのですが、\n",
    "        # それを torch.FloatTensor of size BATCH_SIZEx4 に変換します\n",
    "        # 状態、行動、報酬、non_finalの状態のミニバッチのVariableを作成\n",
    "        # catはConcatenates（結合）のことです。\n",
    "        state_batch = torch.cat(batch.state)\n",
    "        action_batch = torch.cat(batch.action)\n",
    "        reward_batch = torch.cat(batch.reward)\n",
    "        non_final_next_states = torch.cat([s for s in batch.next_state\n",
    "                                           if s is not None])\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 3. 教師信号となるQ(s_t, a_t)値を求める\n",
    "        # -----------------------------------------\n",
    "        # 3.1 ネットワークを推論モードに切り替える\n",
    "        self.model.eval()\n",
    "\n",
    "        # 3.2 ネットワークが出力したQ(s_t, a_t)を求める\n",
    "        # self.model(state_batch)は、右左の両方のQ値を出力しており\n",
    "        # [torch.FloatTensor of size BATCH_SIZEx2]になっている。\n",
    "        # ここから実行したアクションa_tに対応するQ値を求めるため、action_batchで行った行動a_tが右か左かのindexを求め\n",
    "        # それに対応するQ値をgatherでひっぱり出す。\n",
    "        state_action_values = self.model(state_batch).gather(1, action_batch)\n",
    "\n",
    "        # 3.3 max{Q(s_t+1, a)}値を求める。ただし次の状態があるかに注意。\n",
    "\n",
    "        # cartpoleがdoneになっておらず、next_stateがあるかをチェックするインデックスマスクを作成\n",
    "        non_final_mask = torch.ByteTensor(tuple(map(lambda s: s is not None,\n",
    "                                                    batch.next_state)))\n",
    "        # まずは全部0にしておく\n",
    "        next_state_values = torch.zeros(BATCH_SIZE)\n",
    "\n",
    "        # 次の状態があるindexの最大Q値を求める\n",
    "        # 出力にアクセスし、max(1)で列方向の最大値の[値、index]を求めます\n",
    "        # そしてそのQ値（index=0）を出力します\n",
    "        # detachでその値を取り出します\n",
    "        next_state_values[non_final_mask] = self.model(\n",
    "            non_final_next_states).max(1)[0].detach()\n",
    "\n",
    "        # 3.4 教師となるQ(s_t, a_t)値を、Q学習の式から求める\n",
    "        expected_state_action_values = reward_batch + GAMMA * next_state_values\n",
    "\n",
    "        # -----------------------------------------\n",
    "        # 4. 結合パラメータの更新\n",
    "        # -----------------------------------------\n",
    "        # 4.1 ネットワークを訓練モードに切り替える\n",
    "        self.model.train()\n",
    "\n",
    "        # 4.2 損失関数を計算する（smooth_l1_lossはHuberloss）\n",
    "        # expected_state_action_valuesは\n",
    "        # sizeが[minbatch]になっているので、unsqueezeで[minibatch x 1]へ\n",
    "        loss = F.smooth_l1_loss(state_action_values,\n",
    "                                expected_state_action_values.unsqueeze(1))\n",
    "\n",
    "        # 4.3 結合パラメータを更新する\n",
    "        self.optimizer.zero_grad()  # 勾配をリセット\n",
    "        loss.backward()  # バックプロパゲーションを計算\n",
    "        self.optimizer.step()  # 結合パラメータを更新\n",
    "\n",
    "    def decide_action(self, state, episode):\n",
    "        '''現在の状態に応じて、行動を決定する'''\n",
    "        # ε-greedy法で徐々に最適行動のみを採用する\n",
    "        epsilon = 0.5 * (1 / (episode + 1))\n",
    "\n",
    "        if epsilon <= np.random.uniform(0, 1):\n",
    "            self.model.eval()  # ネットワークを推論モードに切り替える\n",
    "            with torch.no_grad():\n",
    "                action = self.model(state).max(1)[1].view(1, 1)\n",
    "            # ネットワークの出力の最大値のindexを取り出します = max(1)[1]\n",
    "            # .view(1,1)は[torch.LongTensor of size 1]　を size 1x1 に変換します\n",
    "\n",
    "        else:\n",
    "            # 0,1の行動をランダムに返す\n",
    "            action = torch.LongTensor(\n",
    "                [[random.randrange(self.num_actions)]])  # 0,1の行動をランダムに返す\n",
    "            # actionは[torch.LongTensor of size 1x1]の形になります\n",
    "\n",
    "        return action\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleで動くエージェントクラスです、棒付き台車そのものになります\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        '''課題の状態と行動の数を設定する'''\n",
    "        self.brain = Brain(num_states, num_actions)  # エージェントが行動を決定するための頭脳を生成\n",
    "\n",
    "    def update_q_function(self):\n",
    "        '''Q関数を更新する'''\n",
    "        self.brain.replay()\n",
    "\n",
    "    def get_action(self, state, episode):\n",
    "        '''行動を決定する'''\n",
    "        action = self.brain.decide_action(state, episode)\n",
    "        return action\n",
    "\n",
    "    def memorize(self, state, action, state_next, reward):\n",
    "        '''memoryオブジェクトに、state, action, state_next, rewardの内容を保存する'''\n",
    "        self.brain.memory.push(state, action, state_next, reward)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleを実行する環境のクラスです\n",
    "\n",
    "\n",
    "class Environment:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.env = gym.make(ENV,render_mode='rgb_array')  # 実行する課題を設定\n",
    "        num_states = self.env.observation_space.shape[0]  # 課題の状態数4を取得\n",
    "        num_actions = self.env.action_space.n  # CartPoleの行動（右に左に押す）の2を取得\n",
    "        self.agent = Agent(num_states, num_actions)  # 環境内で行動するAgentを生成\n",
    "\n",
    "        \n",
    "    def run(self):\n",
    "        '''実行'''\n",
    "        episode_10_list = np.zeros(10)  # 10試行分の立ち続けたstep数を格納し、平均ステップ数を出力に利用\n",
    "        complete_episodes = 0  # 195step以上連続で立ち続けた試行数\n",
    "        episode_final = False  # 最後の試行フラグ\n",
    "        frames = []  # 最後の試行を動画にするために画像を格納する変数\n",
    "\n",
    "        for episode in range(NUM_EPISODES):  # 最大試行数分繰り返す\n",
    "            observation = self.env.reset()[0]  # 環境の初期化\n",
    "\n",
    "            state = observation  # 観測をそのまま状態sとして使用\n",
    "            state = torch.from_numpy(state).type(\n",
    "                torch.FloatTensor)  # NumPy変数をPyTorchのテンソルに変換\n",
    "            state = torch.unsqueeze(state, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "            for step in range(MAX_STEPS):  # 1エピソードのループ\n",
    "\n",
    "                if episode_final is True:  # 最終試行ではframesに各時刻の画像を追加していく\n",
    "                    frames.append(self.env.render())\n",
    "\n",
    "                action = self.agent.get_action(state, episode)  # 行動を求める\n",
    "\n",
    "                # 行動a_tの実行により、s_{t+1}とdoneフラグを求める\n",
    "                # actionから.item()を指定して、中身を取り出す\n",
    "                observation_next, _, done, _, _ = self.env.step(\n",
    "                    action.item())  # rewardとinfoは使わないので_にする\n",
    "\n",
    "                # 報酬を与える。さらにepisodeの終了評価と、state_nextを設定する\n",
    "                if done:  # ステップ数が200経過するか、一定角度以上傾くとdoneはtrueになる\n",
    "                    state_next = None  # 次の状態はないので、Noneを格納\n",
    "\n",
    "                    # 直近10episodeの立てたstep数リストに追加\n",
    "                    episode_10_list = np.hstack(\n",
    "                        (episode_10_list[1:], step + 1))\n",
    "\n",
    "                    if step < 175:\n",
    "                        reward = torch.FloatTensor(\n",
    "                            [-1.0])  # 途中でこけたら罰則として報酬-1を与える\n",
    "                        complete_episodes = 0  # 連続成功記録をリセット\n",
    "                    else:\n",
    "                        reward = torch.FloatTensor([1.0])  # 立ったまま終了時は報酬1を与える\n",
    "                        complete_episodes = complete_episodes + 1  # 連続記録を更新\n",
    "                else:\n",
    "                    reward = torch.FloatTensor([0.0])  # 普段は報酬0\n",
    "                    state_next = observation_next  # 観測をそのまま状態とする\n",
    "                    state_next = torch.from_numpy(state_next).type(\n",
    "                        torch.FloatTensor)  # numpy変数をPyTorchのテンソルに変換\n",
    "                    state_next = torch.unsqueeze(state_next, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "                # メモリに経験を追加\n",
    "                self.agent.memorize(state, action, state_next, reward)\n",
    "\n",
    "                # Experience ReplayでQ関数を更新する\n",
    "                self.agent.update_q_function()\n",
    "\n",
    "                # 観測の更新\n",
    "                state = state_next\n",
    "\n",
    "                # 終了時の処理\n",
    "                if done:\n",
    "                    print('%d Episode: Finished after %d steps：10次试验平均step数 = %.1lf' % (\n",
    "                        episode, step + 1, episode_10_list.mean()))\n",
    "                    break\n",
    "\n",
    "            if episode_final is True:\n",
    "                # 動画を保存と描画\n",
    "                display_frames_as_gif(frames)\n",
    "                break\n",
    "\n",
    "            # 10連続で200step経ち続けたら成功\n",
    "            if complete_episodes >= 10:\n",
    "                print('10回连续成功')\n",
    "                episode_final = True  # 次の試行を描画を行う最終試行とする\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (fc1): Linear(in_features=4, out_features=32, bias=True)\n",
      "  (relu1): ReLU()\n",
      "  (fc2): Linear(in_features=32, out_features=32, bias=True)\n",
      "  (relu2): ReLU()\n",
      "  (fc3): Linear(in_features=32, out_features=2, bias=True)\n",
      ")\n",
      "0 Episode: Finished after 16 steps：10次试验平均step数 = 1.6\n",
      "1 Episode: Finished after 12 steps：10次试验平均step数 = 2.8\n",
      "2 Episode: Finished after 10 steps：10次试验平均step数 = 3.8\n",
      "3 Episode: Finished after 11 steps：10次试验平均step数 = 4.9\n",
      "4 Episode: Finished after 10 steps：10次试验平均step数 = 5.9\n",
      "5 Episode: Finished after 10 steps：10次试验平均step数 = 6.9\n",
      "6 Episode: Finished after 10 steps：10次试验平均step数 = 7.9\n",
      "7 Episode: Finished after 10 steps：10次试验平均step数 = 8.9\n",
      "8 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "9 Episode: Finished after 9 steps：10次试验平均step数 = 10.8\n",
      "10 Episode: Finished after 9 steps：10次试验平均step数 = 10.1\n",
      "11 Episode: Finished after 12 steps：10次试验平均step数 = 10.1\n",
      "12 Episode: Finished after 10 steps：10次试验平均step数 = 10.1\n",
      "13 Episode: Finished after 10 steps：10次试验平均step数 = 10.0\n",
      "14 Episode: Finished after 9 steps：10次试验平均step数 = 9.9\n",
      "15 Episode: Finished after 9 steps：10次试验平均step数 = 9.8\n",
      "16 Episode: Finished after 9 steps：10次试验平均step数 = 9.7\n",
      "17 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "18 Episode: Finished after 11 steps：10次试验平均step数 = 9.7\n",
      "19 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "20 Episode: Finished after 11 steps：10次试验平均step数 = 10.0\n",
      "21 Episode: Finished after 9 steps：10次试验平均step数 = 9.7\n",
      "22 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "23 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "24 Episode: Finished after 11 steps：10次试验平均step数 = 9.9\n",
      "25 Episode: Finished after 8 steps：10次试验平均step数 = 9.8\n",
      "26 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "27 Episode: Finished after 9 steps：10次试验平均step数 = 9.9\n",
      "28 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "29 Episode: Finished after 9 steps：10次试验平均step数 = 9.7\n",
      "30 Episode: Finished after 9 steps：10次试验平均step数 = 9.5\n",
      "31 Episode: Finished after 10 steps：10次试验平均step数 = 9.6\n",
      "32 Episode: Finished after 10 steps：10次试验平均step数 = 9.6\n",
      "33 Episode: Finished after 10 steps：10次试验平均step数 = 9.6\n",
      "34 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "35 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "36 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "37 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "38 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "39 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "40 Episode: Finished after 9 steps：10次试验平均step数 = 9.8\n",
      "41 Episode: Finished after 9 steps：10次试验平均step数 = 9.7\n",
      "42 Episode: Finished after 11 steps：10次试验平均step数 = 9.8\n",
      "43 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "44 Episode: Finished after 9 steps：10次试验平均step数 = 9.7\n",
      "45 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "46 Episode: Finished after 11 steps：10次试验平均step数 = 9.9\n",
      "47 Episode: Finished after 8 steps：10次试验平均step数 = 9.7\n",
      "48 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "49 Episode: Finished after 10 steps：10次试验平均step数 = 9.6\n",
      "50 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "51 Episode: Finished after 10 steps：10次试验平均step数 = 9.7\n",
      "52 Episode: Finished after 9 steps：10次试验平均step数 = 9.5\n",
      "53 Episode: Finished after 9 steps：10次试验平均step数 = 9.4\n",
      "54 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "55 Episode: Finished after 9 steps：10次试验平均step数 = 9.4\n",
      "56 Episode: Finished after 9 steps：10次试验平均step数 = 9.2\n",
      "57 Episode: Finished after 10 steps：10次试验平均step数 = 9.4\n",
      "58 Episode: Finished after 9 steps：10次试验平均step数 = 9.4\n",
      "59 Episode: Finished after 10 steps：10次试验平均step数 = 9.4\n",
      "60 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "61 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "62 Episode: Finished after 11 steps：10次试验平均step数 = 9.7\n",
      "63 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "64 Episode: Finished after 8 steps：10次试验平均step数 = 9.6\n",
      "65 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "66 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "67 Episode: Finished after 11 steps：10次试验平均step数 = 9.7\n",
      "68 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "69 Episode: Finished after 11 steps：10次试验平均step数 = 9.9\n",
      "70 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "71 Episode: Finished after 11 steps：10次试验平均step数 = 10.0\n",
      "72 Episode: Finished after 9 steps：10次试验平均step数 = 9.8\n",
      "73 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "74 Episode: Finished after 12 steps：10次试验平均step数 = 10.2\n",
      "75 Episode: Finished after 10 steps：10次试验平均step数 = 10.3\n",
      "76 Episode: Finished after 11 steps：10次试验平均step数 = 10.5\n",
      "77 Episode: Finished after 10 steps：10次试验平均step数 = 10.4\n",
      "78 Episode: Finished after 10 steps：10次试验平均step数 = 10.4\n",
      "79 Episode: Finished after 10 steps：10次试验平均step数 = 10.3\n",
      "80 Episode: Finished after 9 steps：10次试验平均step数 = 10.2\n",
      "81 Episode: Finished after 10 steps：10次试验平均step数 = 10.1\n",
      "82 Episode: Finished after 10 steps：10次试验平均step数 = 10.2\n",
      "83 Episode: Finished after 10 steps：10次试验平均step数 = 10.2\n",
      "84 Episode: Finished after 8 steps：10次试验平均step数 = 9.8\n",
      "85 Episode: Finished after 10 steps：10次试验平均step数 = 9.8\n",
      "86 Episode: Finished after 9 steps：10次试验平均step数 = 9.6\n",
      "87 Episode: Finished after 9 steps：10次试验平均step数 = 9.5\n",
      "88 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "89 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "90 Episode: Finished after 8 steps：10次试验平均step数 = 9.4\n",
      "91 Episode: Finished after 10 steps：10次试验平均step数 = 9.4\n",
      "92 Episode: Finished after 9 steps：10次试验平均step数 = 9.3\n",
      "93 Episode: Finished after 10 steps：10次试验平均step数 = 9.3\n",
      "94 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "95 Episode: Finished after 10 steps：10次试验平均step数 = 9.5\n",
      "96 Episode: Finished after 12 steps：10次试验平均step数 = 9.8\n",
      "97 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "98 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "99 Episode: Finished after 10 steps：10次试验平均step数 = 9.9\n",
      "100 Episode: Finished after 11 steps：10次试验平均step数 = 10.2\n",
      "101 Episode: Finished after 12 steps：10次试验平均step数 = 10.4\n",
      "103 Episode: Finished after 185 steps：10次试验平均step数 = 28.0\n",
      "104 Episode: Finished after 118 steps：10次试验平均step数 = 38.8\n",
      "105 Episode: Finished after 81 steps：10次试验平均step数 = 45.9\n",
      "106 Episode: Finished after 64 steps：10次试验平均step数 = 51.3\n",
      "107 Episode: Finished after 44 steps：10次试验平均step数 = 54.5\n",
      "108 Episode: Finished after 42 steps：10次试验平均step数 = 57.7\n",
      "109 Episode: Finished after 63 steps：10次试验平均step数 = 63.0\n",
      "110 Episode: Finished after 72 steps：10次试验平均step数 = 69.2\n",
      "111 Episode: Finished after 52 steps：10次试验平均step数 = 73.3\n",
      "112 Episode: Finished after 56 steps：10次试验平均step数 = 77.7\n",
      "113 Episode: Finished after 40 steps：10次试验平均step数 = 63.2\n",
      "114 Episode: Finished after 41 steps：10次试验平均step数 = 55.5\n",
      "115 Episode: Finished after 55 steps：10次试验平均step数 = 52.9\n",
      "116 Episode: Finished after 49 steps：10次试验平均step数 = 51.4\n",
      "117 Episode: Finished after 58 steps：10次试验平均step数 = 52.8\n",
      "118 Episode: Finished after 49 steps：10次试验平均step数 = 53.5\n",
      "119 Episode: Finished after 61 steps：10次试验平均step数 = 53.3\n",
      "120 Episode: Finished after 55 steps：10次试验平均step数 = 51.6\n",
      "121 Episode: Finished after 78 steps：10次试验平均step数 = 54.2\n",
      "122 Episode: Finished after 56 steps：10次试验平均step数 = 54.2\n",
      "123 Episode: Finished after 37 steps：10次试验平均step数 = 53.9\n",
      "124 Episode: Finished after 50 steps：10次试验平均step数 = 54.8\n",
      "125 Episode: Finished after 125 steps：10次试验平均step数 = 61.8\n",
      "126 Episode: Finished after 120 steps：10次试验平均step数 = 68.9\n",
      "127 Episode: Finished after 88 steps：10次试验平均step数 = 71.9\n",
      "128 Episode: Finished after 60 steps：10次试验平均step数 = 73.0\n",
      "129 Episode: Finished after 90 steps：10次试验平均step数 = 75.9\n",
      "130 Episode: Finished after 140 steps：10次试验平均step数 = 84.4\n",
      "131 Episode: Finished after 71 steps：10次试验平均step数 = 83.7\n",
      "132 Episode: Finished after 115 steps：10次试验平均step数 = 89.6\n",
      "133 Episode: Finished after 56 steps：10次试验平均step数 = 91.5\n",
      "134 Episode: Finished after 88 steps：10次试验平均step数 = 95.3\n",
      "135 Episode: Finished after 69 steps：10次试验平均step数 = 89.7\n",
      "136 Episode: Finished after 69 steps：10次试验平均step数 = 84.6\n",
      "137 Episode: Finished after 54 steps：10次试验平均step数 = 81.2\n",
      "138 Episode: Finished after 71 steps：10次试验平均step数 = 82.3\n",
      "139 Episode: Finished after 117 steps：10次试验平均step数 = 85.0\n",
      "140 Episode: Finished after 120 steps：10次试验平均step数 = 83.0\n",
      "141 Episode: Finished after 115 steps：10次试验平均step数 = 87.4\n",
      "142 Episode: Finished after 67 steps：10次试验平均step数 = 82.6\n",
      "143 Episode: Finished after 91 steps：10次试验平均step数 = 86.1\n",
      "144 Episode: Finished after 107 steps：10次试验平均step数 = 88.0\n",
      "145 Episode: Finished after 116 steps：10次试验平均step数 = 92.7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "146 Episode: Finished after 90 steps：10次试验平均step数 = 94.8\n",
      "147 Episode: Finished after 77 steps：10次试验平均step数 = 97.1\n",
      "148 Episode: Finished after 78 steps：10次试验平均step数 = 97.8\n",
      "149 Episode: Finished after 110 steps：10次试验平均step数 = 97.1\n",
      "150 Episode: Finished after 81 steps：10次试验平均step数 = 93.2\n",
      "151 Episode: Finished after 66 steps：10次试验平均step数 = 88.3\n",
      "152 Episode: Finished after 147 steps：10次试验平均step数 = 96.3\n",
      "153 Episode: Finished after 88 steps：10次试验平均step数 = 96.0\n",
      "154 Episode: Finished after 130 steps：10次试验平均step数 = 98.3\n",
      "155 Episode: Finished after 62 steps：10次试验平均step数 = 92.9\n",
      "156 Episode: Finished after 94 steps：10次试验平均step数 = 93.3\n",
      "157 Episode: Finished after 175 steps：10次试验平均step数 = 103.1\n",
      "158 Episode: Finished after 162 steps：10次试验平均step数 = 111.5\n",
      "159 Episode: Finished after 192 steps：10次试验平均step数 = 119.7\n",
      "160 Episode: Finished after 125 steps：10次试验平均step数 = 124.1\n",
      "161 Episode: Finished after 147 steps：10次试验平均step数 = 132.2\n",
      "162 Episode: Finished after 185 steps：10次试验平均step数 = 136.0\n",
      "164 Episode: Finished after 137 steps：10次试验平均step数 = 140.9\n",
      "167 Episode: Finished after 176 steps：10次试验平均step数 = 145.5\n",
      "169 Episode: Finished after 143 steps：10次试验平均step数 = 153.6\n",
      "170 Episode: Finished after 169 steps：10次试验平均step数 = 161.1\n",
      "173 Episode: Finished after 172 steps：10次试验平均step数 = 160.8\n",
      "175 Episode: Finished after 173 steps：10次试验平均step数 = 161.9\n",
      "179 Episode: Finished after 197 steps：10次试验平均step数 = 162.4\n",
      "183 Episode: Finished after 181 steps：10次试验平均step数 = 168.0\n",
      "184 Episode: Finished after 189 steps：10次试验平均step数 = 172.2\n",
      "363 Episode: Finished after 153 steps：10次试验平均step数 = 169.0\n",
      "365 Episode: Finished after 147 steps：10次试验平均step数 = 170.0\n",
      "366 Episode: Finished after 125 steps：10次试验平均step数 = 164.9\n",
      "369 Episode: Finished after 188 steps：10次试验平均step数 = 169.4\n",
      "377 Episode: Finished after 166 steps：10次试验平均step数 = 169.1\n",
      "380 Episode: Finished after 61 steps：10次试验平均step数 = 158.0\n",
      "381 Episode: Finished after 112 steps：10次试验平均step数 = 151.9\n",
      "382 Episode: Finished after 134 steps：10次试验平均step数 = 145.6\n",
      "383 Episode: Finished after 85 steps：10次试验平均step数 = 136.0\n",
      "384 Episode: Finished after 71 steps：10次试验平均step数 = 124.2\n",
      "385 Episode: Finished after 55 steps：10次试验平均step数 = 114.4\n",
      "386 Episode: Finished after 119 steps：10次试验平均step数 = 111.6\n",
      "387 Episode: Finished after 57 steps：10次试验平均step数 = 104.8\n",
      "388 Episode: Finished after 154 steps：10次试验平均step数 = 101.4\n",
      "390 Episode: Finished after 88 steps：10次试验平均step数 = 93.6\n",
      "391 Episode: Finished after 92 steps：10次试验平均step数 = 96.7\n",
      "392 Episode: Finished after 151 steps：10次试验平均step数 = 100.6\n",
      "393 Episode: Finished after 137 steps：10次试验平均step数 = 100.9\n",
      "395 Episode: Finished after 110 steps：10次试验平均step数 = 103.4\n",
      "396 Episode: Finished after 120 steps：10次试验平均step数 = 108.3\n",
      "397 Episode: Finished after 119 steps：10次试验平均step数 = 114.7\n",
      "398 Episode: Finished after 154 steps：10次试验平均step数 = 118.2\n",
      "399 Episode: Finished after 177 steps：10次试验平均step数 = 130.2\n",
      "401 Episode: Finished after 191 steps：10次试验平均step数 = 133.9\n",
      "402 Episode: Finished after 151 steps：10次试验平均step数 = 140.2\n",
      "404 Episode: Finished after 194 steps：10次试验平均step数 = 150.4\n",
      "406 Episode: Finished after 176 steps：10次试验平均step数 = 152.9\n",
      "407 Episode: Finished after 162 steps：10次试验平均step数 = 155.4\n",
      "408 Episode: Finished after 155 steps：10次试验平均step数 = 159.9\n",
      "409 Episode: Finished after 170 steps：10次试验平均step数 = 164.9\n",
      "410 Episode: Finished after 187 steps：10次试验平均step数 = 171.7\n",
      "411 Episode: Finished after 161 steps：10次试验平均step数 = 172.4\n",
      "412 Episode: Finished after 191 steps：10次试验平均step数 = 173.8\n",
      "419 Episode: Finished after 189 steps：10次试验平均step数 = 173.6\n",
      "422 Episode: Finished after 164 steps：10次试验平均step数 = 174.9\n",
      "423 Episode: Finished after 136 steps：10次试验平均step数 = 169.1\n",
      "425 Episode: Finished after 164 steps：10次试验平均step数 = 167.9\n",
      "427 Episode: Finished after 195 steps：10次试验平均step数 = 171.2\n",
      "429 Episode: Finished after 184 steps：10次试验平均step数 = 174.1\n",
      "430 Episode: Finished after 153 steps：10次试验平均step数 = 172.4\n",
      "431 Episode: Finished after 166 steps：10次试验平均step数 = 170.3\n",
      "432 Episode: Finished after 150 steps：10次试验平均step数 = 169.2\n",
      "438 Episode: Finished after 179 steps：10次试验平均step数 = 168.0\n",
      "441 Episode: Finished after 197 steps：10次试验平均step数 = 168.8\n",
      "442 Episode: Finished after 155 steps：10次试验平均step数 = 167.9\n",
      "443 Episode: Finished after 156 steps：10次试验平均step数 = 169.9\n",
      "444 Episode: Finished after 179 steps：10次试验平均step数 = 171.4\n",
      "445 Episode: Finished after 180 steps：10次试验平均step数 = 169.9\n",
      "446 Episode: Finished after 178 steps：10次试验平均step数 = 169.3\n",
      "447 Episode: Finished after 185 steps：10次试验平均step数 = 172.5\n",
      "448 Episode: Finished after 193 steps：10次试验平均step数 = 175.2\n",
      "450 Episode: Finished after 162 steps：10次试验平均step数 = 176.4\n",
      "455 Episode: Finished after 186 steps：10次试验平均step数 = 177.1\n",
      "456 Episode: Finished after 183 steps：10次试验平均step数 = 175.7\n",
      "491 Episode: Finished after 189 steps：10次试验平均step数 = 179.1\n",
      "691 Episode: Finished after 119 steps：10次试验平均step数 = 175.4\n",
      "696 Episode: Finished after 163 steps：10次试验平均step数 = 173.8\n",
      "723 Episode: Finished after 108 steps：10次试验平均step数 = 166.6\n",
      "734 Episode: Finished after 87 steps：10次试验平均step数 = 157.5\n",
      "737 Episode: Finished after 139 steps：10次试验平均step数 = 152.9\n",
      "738 Episode: Finished after 189 steps：10次试验平均step数 = 152.5\n",
      "739 Episode: Finished after 90 steps：10次试验平均step数 = 145.3\n",
      "741 Episode: Finished after 129 steps：10次试验平均step数 = 139.6\n",
      "744 Episode: Finished after 189 steps：10次试验平均step数 = 140.2\n",
      "745 Episode: Finished after 154 steps：10次试验平均step数 = 136.7\n",
      "746 Episode: Finished after 160 steps：10次试验平均step数 = 140.8\n",
      "747 Episode: Finished after 118 steps：10次试验平均step数 = 136.3\n",
      "748 Episode: Finished after 199 steps：10次试验平均step数 = 145.4\n",
      "749 Episode: Finished after 160 steps：10次试验平均step数 = 152.7\n",
      "750 Episode: Finished after 131 steps：10次试验平均step数 = 151.9\n",
      "751 Episode: Finished after 153 steps：10次试验平均step数 = 148.3\n",
      "752 Episode: Finished after 98 steps：10次试验平均step数 = 149.1\n",
      "753 Episode: Finished after 129 steps：10次试验平均step数 = 149.1\n",
      "754 Episode: Finished after 121 steps：10次试验平均step数 = 142.3\n",
      "756 Episode: Finished after 106 steps：10次试验平均step数 = 137.5\n",
      "757 Episode: Finished after 122 steps：10次试验平均step数 = 133.7\n",
      "758 Episode: Finished after 115 steps：10次试验平均step数 = 133.4\n",
      "759 Episode: Finished after 66 steps：10次试验平均step数 = 120.1\n",
      "760 Episode: Finished after 59 steps：10次试验平均step数 = 110.0\n",
      "761 Episode: Finished after 140 steps：10次试验平均step数 = 110.9\n",
      "762 Episode: Finished after 61 steps：10次试验平均step数 = 101.7\n",
      "763 Episode: Finished after 32 steps：10次试验平均step数 = 95.1\n",
      "764 Episode: Finished after 120 steps：10次试验平均step数 = 94.2\n",
      "765 Episode: Finished after 59 steps：10次试验平均step数 = 88.0\n",
      "766 Episode: Finished after 113 steps：10次试验平均step数 = 88.7\n",
      "767 Episode: Finished after 114 steps：10次试验平均step数 = 87.9\n",
      "768 Episode: Finished after 170 steps：10次试验平均step数 = 93.4\n",
      "769 Episode: Finished after 108 steps：10次试验平均step数 = 97.6\n",
      "770 Episode: Finished after 71 steps：10次试验平均step数 = 98.8\n",
      "771 Episode: Finished after 130 steps：10次试验平均step数 = 97.8\n",
      "772 Episode: Finished after 137 steps：10次试验平均step数 = 105.4\n",
      "773 Episode: Finished after 113 steps：10次试验平均step数 = 113.5\n",
      "774 Episode: Finished after 67 steps：10次试验平均step数 = 108.2\n",
      "775 Episode: Finished after 87 steps：10次试验平均step数 = 111.0\n",
      "776 Episode: Finished after 40 steps：10次试验平均step数 = 103.7\n",
      "778 Episode: Finished after 161 steps：10次试验平均step数 = 108.4\n",
      "779 Episode: Finished after 20 steps：10次试验平均step数 = 93.4\n",
      "780 Episode: Finished after 30 steps：10次试验平均step数 = 85.6\n",
      "781 Episode: Finished after 133 steps：10次试验平均step数 = 91.8\n",
      "782 Episode: Finished after 157 steps：10次试验平均step数 = 94.5\n",
      "783 Episode: Finished after 121 steps：10次试验平均step数 = 92.9\n",
      "784 Episode: Finished after 108 steps：10次试验平均step数 = 92.4\n",
      "786 Episode: Finished after 145 steps：10次试验平均step数 = 100.2\n",
      "788 Episode: Finished after 149 steps：10次试验平均step数 = 106.4\n",
      "789 Episode: Finished after 175 steps：10次试验平均step数 = 119.9\n",
      "790 Episode: Finished after 150 steps：10次试验平均step数 = 118.8\n",
      "791 Episode: Finished after 188 steps：10次试验平均step数 = 135.6\n",
      "792 Episode: Finished after 137 steps：10次试验平均step数 = 146.3\n",
      "793 Episode: Finished after 156 steps：10次试验平均step数 = 148.6\n",
      "794 Episode: Finished after 124 steps：10次试验平均step数 = 145.3\n",
      "795 Episode: Finished after 145 steps：10次试验平均step数 = 147.7\n",
      "796 Episode: Finished after 200 steps：10次试验平均step数 = 156.9\n",
      "797 Episode: Finished after 190 steps：10次试验平均step数 = 161.4\n",
      "799 Episode: Finished after 178 steps：10次试验平均step数 = 164.3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "801 Episode: Finished after 156 steps：10次试验平均step数 = 162.4\n",
      "802 Episode: Finished after 173 steps：10次试验平均step数 = 164.7\n",
      "803 Episode: Finished after 185 steps：10次试验平均step数 = 164.4\n",
      "805 Episode: Finished after 154 steps：10次试验平均step数 = 166.1\n",
      "806 Episode: Finished after 156 steps：10次试验平均step数 = 166.1\n",
      "807 Episode: Finished after 154 steps：10次试验平均step数 = 169.1\n",
      "809 Episode: Finished after 134 steps：10次试验平均step数 = 168.0\n",
      "810 Episode: Finished after 175 steps：10次试验平均step数 = 165.5\n",
      "811 Episode: Finished after 195 steps：10次试验平均step数 = 166.0\n",
      "812 Episode: Finished after 157 steps：10次试验平均step数 = 163.9\n",
      "813 Episode: Finished after 199 steps：10次试验平均step数 = 168.2\n",
      "814 Episode: Finished after 133 steps：10次试验平均step数 = 164.2\n",
      "815 Episode: Finished after 188 steps：10次试验平均step数 = 164.5\n",
      "816 Episode: Finished after 152 steps：10次试验平均step数 = 164.3\n",
      "818 Episode: Finished after 167 steps：10次试验平均step数 = 165.4\n",
      "819 Episode: Finished after 165 steps：10次试验平均step数 = 166.5\n",
      "820 Episode: Finished after 145 steps：10次试验平均step数 = 167.6\n",
      "821 Episode: Finished after 138 steps：10次试验平均step数 = 163.9\n",
      "822 Episode: Finished after 146 steps：10次试验平均step数 = 159.0\n",
      "823 Episode: Finished after 184 steps：10次试验平均step数 = 161.7\n",
      "824 Episode: Finished after 157 steps：10次试验平均step数 = 157.5\n",
      "825 Episode: Finished after 169 steps：10次试验平均step数 = 161.1\n",
      "826 Episode: Finished after 164 steps：10次试验平均step数 = 158.7\n",
      "827 Episode: Finished after 141 steps：10次试验平均step数 = 157.6\n",
      "828 Episode: Finished after 161 steps：10次试验平均step数 = 157.0\n",
      "829 Episode: Finished after 157 steps：10次试验平均step数 = 156.2\n",
      "830 Episode: Finished after 164 steps：10次试验平均step数 = 158.1\n",
      "831 Episode: Finished after 136 steps：10次试验平均step数 = 157.9\n",
      "832 Episode: Finished after 174 steps：10次试验平均step数 = 160.7\n",
      "833 Episode: Finished after 136 steps：10次试验平均step数 = 155.9\n",
      "834 Episode: Finished after 116 steps：10次试验平均step数 = 151.8\n",
      "835 Episode: Finished after 126 steps：10次试验平均step数 = 147.5\n",
      "836 Episode: Finished after 127 steps：10次试验平均step数 = 143.8\n",
      "837 Episode: Finished after 141 steps：10次试验平均step数 = 143.8\n",
      "838 Episode: Finished after 131 steps：10次试验平均step数 = 140.8\n",
      "839 Episode: Finished after 146 steps：10次试验平均step数 = 139.7\n",
      "840 Episode: Finished after 138 steps：10次试验平均step数 = 137.1\n",
      "841 Episode: Finished after 142 steps：10次试验平均step数 = 137.7\n",
      "842 Episode: Finished after 136 steps：10次试验平均step数 = 133.9\n",
      "843 Episode: Finished after 172 steps：10次试验平均step数 = 137.5\n",
      "844 Episode: Finished after 138 steps：10次试验平均step数 = 139.7\n",
      "845 Episode: Finished after 166 steps：10次试验平均step数 = 143.7\n",
      "846 Episode: Finished after 183 steps：10次试验平均step数 = 149.3\n",
      "847 Episode: Finished after 200 steps：10次试验平均step数 = 155.2\n",
      "848 Episode: Finished after 186 steps：10次试验平均step数 = 160.7\n",
      "849 Episode: Finished after 152 steps：10次试验平均step数 = 161.3\n",
      "850 Episode: Finished after 169 steps：10次试验平均step数 = 164.4\n",
      "851 Episode: Finished after 142 steps：10次试验平均step数 = 164.4\n",
      "852 Episode: Finished after 138 steps：10次试验平均step数 = 164.6\n",
      "853 Episode: Finished after 138 steps：10次试验平均step数 = 161.2\n",
      "854 Episode: Finished after 140 steps：10次试验平均step数 = 161.4\n",
      "855 Episode: Finished after 150 steps：10次试验平均step数 = 159.8\n",
      "856 Episode: Finished after 148 steps：10次试验平均step数 = 156.3\n",
      "857 Episode: Finished after 148 steps：10次试验平均step数 = 151.1\n",
      "858 Episode: Finished after 155 steps：10次试验平均step数 = 148.0\n",
      "859 Episode: Finished after 136 steps：10次试验平均step数 = 146.4\n",
      "860 Episode: Finished after 144 steps：10次试验平均step数 = 143.9\n",
      "861 Episode: Finished after 138 steps：10次试验平均step数 = 143.5\n",
      "862 Episode: Finished after 136 steps：10次试验平均step数 = 143.3\n",
      "863 Episode: Finished after 148 steps：10次试验平均step数 = 144.3\n",
      "864 Episode: Finished after 131 steps：10次试验平均step数 = 143.4\n",
      "865 Episode: Finished after 139 steps：10次试验平均step数 = 142.3\n",
      "866 Episode: Finished after 133 steps：10次试验平均step数 = 140.8\n",
      "867 Episode: Finished after 174 steps：10次试验平均step数 = 143.4\n",
      "868 Episode: Finished after 196 steps：10次试验平均step数 = 147.5\n",
      "869 Episode: Finished after 146 steps：10次试验平均step数 = 148.5\n",
      "870 Episode: Finished after 140 steps：10次试验平均step数 = 148.1\n",
      "871 Episode: Finished after 156 steps：10次试验平均step数 = 149.9\n",
      "872 Episode: Finished after 140 steps：10次试验平均step数 = 150.3\n",
      "873 Episode: Finished after 138 steps：10次试验平均step数 = 149.3\n",
      "874 Episode: Finished after 182 steps：10次试验平均step数 = 154.4\n",
      "875 Episode: Finished after 165 steps：10次试验平均step数 = 157.0\n",
      "876 Episode: Finished after 143 steps：10次试验平均step数 = 158.0\n",
      "877 Episode: Finished after 132 steps：10次试验平均step数 = 153.8\n",
      "878 Episode: Finished after 142 steps：10次试验平均step数 = 148.4\n",
      "879 Episode: Finished after 138 steps：10次试验平均step数 = 147.6\n",
      "880 Episode: Finished after 153 steps：10次试验平均step数 = 148.9\n",
      "881 Episode: Finished after 170 steps：10次试验平均step数 = 150.3\n",
      "882 Episode: Finished after 151 steps：10次试验平均step数 = 151.4\n",
      "883 Episode: Finished after 144 steps：10次试验平均step数 = 152.0\n",
      "884 Episode: Finished after 148 steps：10次试验平均step数 = 148.6\n",
      "885 Episode: Finished after 150 steps：10次试验平均step数 = 147.1\n",
      "886 Episode: Finished after 141 steps：10次试验平均step数 = 146.9\n",
      "887 Episode: Finished after 145 steps：10次试验平均step数 = 148.2\n",
      "888 Episode: Finished after 136 steps：10次试验平均step数 = 147.6\n",
      "889 Episode: Finished after 175 steps：10次试验平均step数 = 151.3\n",
      "890 Episode: Finished after 165 steps：10次试验平均step数 = 152.5\n",
      "891 Episode: Finished after 160 steps：10次试验平均step数 = 151.5\n",
      "892 Episode: Finished after 181 steps：10次试验平均step数 = 154.5\n",
      "894 Episode: Finished after 149 steps：10次试验平均step数 = 155.0\n",
      "895 Episode: Finished after 150 steps：10次试验平均step数 = 155.2\n",
      "896 Episode: Finished after 164 steps：10次试验平均step数 = 156.6\n",
      "897 Episode: Finished after 154 steps：10次试验平均step数 = 157.9\n",
      "898 Episode: Finished after 144 steps：10次试验平均step数 = 157.8\n",
      "899 Episode: Finished after 145 steps：10次试验平均step数 = 158.7\n",
      "900 Episode: Finished after 137 steps：10次试验平均step数 = 154.9\n",
      "901 Episode: Finished after 131 steps：10次试验平均step数 = 151.5\n",
      "902 Episode: Finished after 181 steps：10次试验平均step数 = 153.6\n",
      "903 Episode: Finished after 138 steps：10次试验平均step数 = 149.3\n",
      "904 Episode: Finished after 176 steps：10次试验平均step数 = 152.0\n",
      "905 Episode: Finished after 145 steps：10次试验平均step数 = 151.5\n",
      "906 Episode: Finished after 133 steps：10次试验平均step数 = 148.4\n",
      "907 Episode: Finished after 147 steps：10次试验平均step数 = 147.7\n",
      "908 Episode: Finished after 143 steps：10次试验平均step数 = 147.6\n",
      "909 Episode: Finished after 152 steps：10次试验平均step数 = 148.3\n",
      "910 Episode: Finished after 145 steps：10次试验平均step数 = 149.1\n",
      "911 Episode: Finished after 171 steps：10次试验平均step数 = 153.1\n",
      "912 Episode: Finished after 151 steps：10次试验平均step数 = 150.1\n",
      "913 Episode: Finished after 175 steps：10次试验平均step数 = 153.8\n",
      "914 Episode: Finished after 157 steps：10次试验平均step数 = 151.9\n",
      "915 Episode: Finished after 187 steps：10次试验平均step数 = 156.1\n",
      "916 Episode: Finished after 186 steps：10次试验平均step数 = 161.4\n",
      "917 Episode: Finished after 177 steps：10次试验平均step数 = 164.4\n",
      "918 Episode: Finished after 162 steps：10次试验平均step数 = 166.3\n",
      "919 Episode: Finished after 139 steps：10次试验平均step数 = 165.0\n",
      "920 Episode: Finished after 161 steps：10次试验平均step数 = 166.6\n",
      "921 Episode: Finished after 185 steps：10次试验平均step数 = 168.0\n",
      "922 Episode: Finished after 154 steps：10次试验平均step数 = 168.3\n",
      "923 Episode: Finished after 141 steps：10次试验平均step数 = 164.9\n",
      "924 Episode: Finished after 159 steps：10次试验平均step数 = 165.1\n",
      "925 Episode: Finished after 128 steps：10次试验平均step数 = 159.2\n",
      "926 Episode: Finished after 159 steps：10次试验平均step数 = 156.5\n",
      "927 Episode: Finished after 143 steps：10次试验平均step数 = 153.1\n",
      "928 Episode: Finished after 140 steps：10次试验平均step数 = 150.9\n",
      "929 Episode: Finished after 145 steps：10次试验平均step数 = 151.5\n",
      "930 Episode: Finished after 159 steps：10次试验平均step数 = 151.3\n",
      "931 Episode: Finished after 147 steps：10次试验平均step数 = 147.5\n",
      "932 Episode: Finished after 156 steps：10次试验平均step数 = 147.7\n",
      "933 Episode: Finished after 152 steps：10次试验平均step数 = 148.8\n",
      "934 Episode: Finished after 149 steps：10次试验平均step数 = 147.8\n",
      "935 Episode: Finished after 160 steps：10次试验平均step数 = 151.0\n",
      "936 Episode: Finished after 151 steps：10次试验平均step数 = 150.2\n",
      "937 Episode: Finished after 153 steps：10次试验平均step数 = 151.2\n",
      "938 Episode: Finished after 141 steps：10次试验平均step数 = 151.3\n",
      "939 Episode: Finished after 131 steps：10次试验平均step数 = 149.9\n",
      "940 Episode: Finished after 134 steps：10次试验平均step数 = 147.4\n",
      "941 Episode: Finished after 149 steps：10次试验平均step数 = 147.6\n",
      "942 Episode: Finished after 181 steps：10次试验平均step数 = 150.1\n",
      "943 Episode: Finished after 159 steps：10次试验平均step数 = 150.8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "944 Episode: Finished after 145 steps：10次试验平均step数 = 150.4\n",
      "945 Episode: Finished after 150 steps：10次试验平均step数 = 149.4\n",
      "946 Episode: Finished after 150 steps：10次试验平均step数 = 149.3\n",
      "947 Episode: Finished after 149 steps：10次试验平均step数 = 148.9\n",
      "948 Episode: Finished after 152 steps：10次试验平均step数 = 150.0\n",
      "949 Episode: Finished after 161 steps：10次试验平均step数 = 153.0\n",
      "950 Episode: Finished after 143 steps：10次试验平均step数 = 153.9\n",
      "951 Episode: Finished after 162 steps：10次试验平均step数 = 155.2\n",
      "952 Episode: Finished after 188 steps：10次试验平均step数 = 155.9\n",
      "953 Episode: Finished after 160 steps：10次试验平均step数 = 156.0\n",
      "954 Episode: Finished after 139 steps：10次试验平均step数 = 155.4\n",
      "955 Episode: Finished after 172 steps：10次试验平均step数 = 157.6\n",
      "956 Episode: Finished after 141 steps：10次试验平均step数 = 156.7\n",
      "957 Episode: Finished after 194 steps：10次试验平均step数 = 161.2\n",
      "958 Episode: Finished after 144 steps：10次试验平均step数 = 160.4\n",
      "959 Episode: Finished after 141 steps：10次试验平均step数 = 158.4\n",
      "960 Episode: Finished after 162 steps：10次试验平均step数 = 160.3\n",
      "961 Episode: Finished after 160 steps：10次试验平均step数 = 160.1\n",
      "962 Episode: Finished after 157 steps：10次试验平均step数 = 157.0\n",
      "963 Episode: Finished after 148 steps：10次试验平均step数 = 155.8\n",
      "964 Episode: Finished after 159 steps：10次试验平均step数 = 157.8\n",
      "965 Episode: Finished after 137 steps：10次试验平均step数 = 154.3\n",
      "966 Episode: Finished after 173 steps：10次试验平均step数 = 157.5\n",
      "967 Episode: Finished after 152 steps：10次试验平均step数 = 153.3\n",
      "968 Episode: Finished after 188 steps：10次试验平均step数 = 157.7\n",
      "969 Episode: Finished after 144 steps：10次试验平均step数 = 158.0\n",
      "970 Episode: Finished after 191 steps：10次试验平均step数 = 160.9\n",
      "971 Episode: Finished after 146 steps：10次试验平均step数 = 159.5\n",
      "972 Episode: Finished after 148 steps：10次试验平均step数 = 158.6\n",
      "973 Episode: Finished after 174 steps：10次试验平均step数 = 161.2\n",
      "974 Episode: Finished after 161 steps：10次试验平均step数 = 161.4\n",
      "975 Episode: Finished after 180 steps：10次试验平均step数 = 165.7\n",
      "976 Episode: Finished after 155 steps：10次试验平均step数 = 163.9\n",
      "977 Episode: Finished after 144 steps：10次试验平均step数 = 163.1\n",
      "978 Episode: Finished after 196 steps：10次试验平均step数 = 163.9\n",
      "979 Episode: Finished after 158 steps：10次试验平均step数 = 165.3\n",
      "980 Episode: Finished after 189 steps：10次试验平均step数 = 165.1\n",
      "981 Episode: Finished after 181 steps：10次试验平均step数 = 168.6\n",
      "982 Episode: Finished after 170 steps：10次试验平均step数 = 170.8\n",
      "983 Episode: Finished after 159 steps：10次试验平均step数 = 169.3\n",
      "985 Episode: Finished after 156 steps：10次试验平均step数 = 168.8\n",
      "986 Episode: Finished after 184 steps：10次试验平均step数 = 169.2\n",
      "987 Episode: Finished after 193 steps：10次试验平均step数 = 173.0\n",
      "988 Episode: Finished after 189 steps：10次试验平均step数 = 177.5\n",
      "989 Episode: Finished after 167 steps：10次试验平均step数 = 174.6\n",
      "990 Episode: Finished after 144 steps：10次试验平均step数 = 173.2\n",
      "991 Episode: Finished after 172 steps：10次试验平均step数 = 171.5\n",
      "992 Episode: Finished after 200 steps：10次试验平均step数 = 173.4\n",
      "993 Episode: Finished after 141 steps：10次试验平均step数 = 170.5\n",
      "994 Episode: Finished after 153 steps：10次试验平均step数 = 169.9\n",
      "995 Episode: Finished after 146 steps：10次试验平均step数 = 168.9\n",
      "996 Episode: Finished after 179 steps：10次试验平均step数 = 168.4\n",
      "997 Episode: Finished after 187 steps：10次试验平均step数 = 167.8\n",
      "998 Episode: Finished after 155 steps：10次试验平均step数 = 164.4\n",
      "999 Episode: Finished after 152 steps：10次试验平均step数 = 162.9\n"
     ]
    }
   ],
   "source": [
    "# main クラス\n",
    "cartpole_env = Environment()\n",
    "cartpole_env.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
